package com.chance.cc.crawler.development.bootstrap.xcar;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.downloader.HttpConfig;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.downloader.HttpRequest;
import com.chance.cc.crawler.core.downloader.http.HttpClientDownloader;
import com.chance.cc.crawler.core.downloader.http.HttpClientGenerator;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.downloader.proxy.SiteConfigProxyProvider;
import com.chance.cc.crawler.core.selector.Html;
import com.chance.cc.crawler.core.selector.Selectable;
import com.chance.cc.crawler.development.utils.RedisReader;
import groovy.util.logging.Slf4j;
import jdk.nashorn.api.scripting.NashornScriptEngineFactory;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.script.*;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @author bx
 * @date 2021/1/8 0008 15:37
 */
@Slf4j
public class XCarHttpTest {
    public static Logger logger = LoggerFactory.getLogger(XCarHttpTest.class);

    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
//        proxy.setUsername("HL89Q19E86E2987D");
//        proxy.setPassword("71F33D94CE5F7BF2");
        proxy.setUsername("H7307T4706B25G4D");
        proxy.setPassword("05B4877CC39192C0");
    }

    public static void main(String[] args) {

        HttpConfig httpConfig = HttpConfig.me("test");
//        httpConfig.setProxy(proxy);
//        httpConfig.setUseProxy(true);

        HttpClientDownloader downloader = new HttpClientDownloader();
        downloader.setClientGenerator(new HttpClientGenerator());
        downloader.setProxyProvider(new SiteConfigProxyProvider());


        for (int i=1;i<=1;i++){
            new Thread(new TestForumRun(downloader,httpConfig),"thread_"+i).start();
        }
    }

    static class TestRun implements Runnable{

        HttpClientDownloader downloader;
        HttpConfig httpConfig;

        public TestRun(HttpClientDownloader downloader, HttpConfig httpConfig) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
        }

        @Override
        public void run() {
            String url = "https://xtv.xcar.com.cn/all/cid701/";
            HttpRequest httpRequest = new HttpRequest();

            for(int i=0;i<1;i++){
                httpRequest.setUrl(String.format(url,Math.random()));
                HttpPage httpPage = downloader.download(httpRequest, httpConfig);
                System.out.println(httpPage);
                Html responseHtml = httpPage.getHtml();
                List<Selectable> nodes = responseHtml.xpath("//div[@class=\"list_mod\"]//dl").nodes();
                for (Selectable node : nodes) {

                    String articleUrl = node.xpath(".//a/@href").get();
                    String releaseTime = node.xpath(".//span[@class=\"time\"]").get();


                    System.out.println(releaseTime);
                }

                RedisReader redisReader = new RedisReader("192.168.1.215",6379,3);
                redisReader.del( "list","map");
//                System.out.println(s);
            }
        }
    }

    static class TestArticleRun implements Runnable{

        HttpClientDownloader downloader;
        HttpConfig httpConfig;

        public TestArticleRun(HttpClientDownloader downloader, HttpConfig httpConfig) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
        }

        @Override
        public void run() {
            String url = "https://info.xcar.com.cn/202101/news_2055414_1.html?viewtype=all";
            HttpRequest httpRequest = new HttpRequest();

            for(int i=0;i<1;i++){
                httpRequest.setUrl(String.format(url,Math.random()));
                HttpPage httpPage = downloader.download(httpRequest, httpConfig);

                Matcher mtAuthor = Pattern.compile("data: \\{([^}])*\\}").matcher(httpPage.getRawText());
                mtAuthor.find();
                String data = mtAuthor.group();
                System.out.println(data.substring(data.indexOf("data: ")+"data: ".length()));
            }
        }
    }

    static class TestKoubeiRun implements Runnable{

        HttpClientDownloader downloader;
        HttpConfig httpConfig;

        public TestKoubeiRun(HttpClientDownloader downloader, HttpConfig httpConfig) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
        }

        @Override
        public void run() {
            String url = "https://newcar.xcar.com.cn/auto/index.php?r=reputation/reputation/GetAjaxKbList3&page=1&pserid=8&jh=0&wd=0";
            HttpRequest httpRequest = new HttpRequest();
            httpRequest.addHeader("user-agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36");
            httpRequest.addHeader("cookie","nguv=c_160999737235228227110117218341611798; _Xdwuv=5ff69c3ce5e31; _Xdwnewuv=1; _PVXuv=5ff69c3df8614; __jsluid_s=49e976754d92ae5b3a1f9be9e2f407db; __jsluid_h=ff270f9a2198fb6530f6e9e06bae425a; __clickidc=161000895164367053; fw_slc=1%3A1610601454%3B1%3A1610601458%3B1%3A1610601459%3B1%3A1610601461%3B1%3A1610601462; fw_pvc=1%3A1610601300%3B1%3A1610601458%3B1%3A1610601461%3B1%3A1610601667%3B1%3A1610601941; fw_clc=1%3A1610601456%3B1%3A1610601460%3B1%3A1610601666%3B1%3A1610601940%3B1%3A1610601992; fw_exc=1%3A1610601456%3B1%3A1610601457%3B1%3A1610601671%3B1%3A1610603232%3B1%3A1610603244; Hm_lvt_53eb54d089f7b5dd4ae2927686b183e0=1610247261,1610333953,1610601302,1610676357; place_prid_lin=2; place_crid_lin=507; place_prname=%E4%B8%8A%E6%B5%B7%E5%B8%82; iwt_uuid=55c322d6-dcca-4fb4-8174-19f8fadc38eb; _locationInfo_=%7Burl%3A%22http%3A%2F%2Fsh.xcar.com.cn%2F%22%2Ccity_id%3A%22507%22%2Cprovince_id%3A%222%22%2C%20city_name%3A%22%25E4%25B8%258A%25E6%25B5%25B7%22%7D; isRemoveCookie=1; place_crname=%E4%B8%8A%E6%B5%B7%E8%BD%A6%E5%B8%82; uv_firstv_refers=https%3A//xtv.xcar.com.cn/show/202101/id_228119.html; __jsl_clearance_s=1610696223.082|0|olPDY0Wa3k3aAbSDhBWQ8W%2BY8UE%3D; zg_did=%7B%22did%22%3A%20%22176db52511822c-0749a9c4fbd349-c791039-100200-176db525119196%22%7D; zg_8f3d0255011c4bc5bae66beca6584825=%7B%22sid%22%3A%201610696223475%2C%22updated%22%3A%201610696260422%2C%22info%22%3A%201610603234040%2C%22superProperty%22%3A%20%22%7B%5C%22platform_type%5C%22%3A%20%5C%22PC%5C%22%2C%5C%22login_id%5C%22%3A%20null%2C%5C%22project_name%5C%22%3A%20%5C%22XCAR%5C%22%2C%5C%22login_status%5C%22%3A%200%7D%22%2C%22platform%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22www.xcar.com.cn%22%2C%22landHref%22%3A%20%22https%3A%2F%2Fxtv.xcar.com.cn%2F%22%2C%22zs%22%3A%200%2C%22sc%22%3A%200%2C%22firstScreen%22%3A%201610696223475%7D; _Xdwstime=1610696261; Hm_lpvt_53eb54d089f7b5dd4ae2927686b183e0=1610696262");

            for(int i=0;i<100;i++){
                List<Map<String,String>> crawlerDataList = new ArrayList<>();

                httpRequest.setUrl(String.format(url,Math.random()));
                HttpPage httpPage = downloader.download(httpRequest, httpConfig);
                Html responseHtml = httpPage.getHtml();
                List<Selectable> nodes = responseHtml.xpath("//div[@class=\"home_list clearfix\"]").nodes();
                for(Selectable node : nodes){
                    Map<String,String> crawlerDataMap = new HashMap<>();
                    listInfos(node,crawlerDataMap);
                    userScoreInfos(node,crawlerDataMap);
                    otherInfos(node,crawlerDataMap);
                    crawlerDataList.add(crawlerDataMap);
                }
                System.out.println(httpPage.getStatusCode() + "-->" + JSON.toJSONString(crawlerDataList));

            }

        }

        private void listInfos(Selectable node,Map<String,String> crawlerDataMap){
            List<Selectable> listInforNodes = node.xpath(".//div[@class=\"list_infor\"]/dl[@class=\"clearfix\"]").nodes();
            for (Selectable listInforNode : listInforNodes) {
                String desc = listInforNode.xpath("./dt").get();
                switch (desc){
                    case "爱车车型":
                        crawlerDataMap.put("model_url","https://newcar.xcar.com.cn"+listInforNode.xpath(".//a/@href").get());
                        crawlerDataMap.put("model",listInforNode.xpath(".//a/text()").get());
                    case "购入时间":
                        crawlerDataMap.put("time_buy",listInforNode.xpath("./dd/text()").get());
                    case "购车地点":
                        crawlerDataMap.put("vendor",listInforNode.xpath("./dd/text()").get());
                    case "裸车价格":
                        crawlerDataMap.put("buy_price",listInforNode.xpath("./dd/text()").get());
                    case "当前油耗":
                        crawlerDataMap.put("fuel_economy",listInforNode.xpath("./dd/text()").get());
                }
            }
        }

        private void userScoreInfos(Selectable node,Map<String,String> crawlerDataMap){
            String userName = node.xpath(".//div[@class=\"name_lf\"]/dl[@class=\"clearfix\"]//span/text()").get();
            crawlerDataMap.put("author",userName);

            String scoreTit = node.xpath(".//div[@class=\"name_lf\"]//div[@class=\"score_tit\"]/text()").get()+
                    node.xpath(".//div[@class=\"name_lf\"]//div[@class=\"score_tit\"]/em/text()").get();
            StringBuffer scoreBuffer = new StringBuffer(scoreTit);
            List<Selectable> scoreNodeList = node.xpath(".//div[@class=\"name_lf\"]//ul[@class=\"score_list\"]/li").nodes();
            for (Selectable scoreNodeInfo : scoreNodeList) {
                String score = scoreNodeInfo.xpath("./span/text()").get() + scoreNodeInfo.xpath("./em/text()").get();
                scoreBuffer.append("\n").append(score);
            }
            crawlerDataMap.put("values",scoreBuffer.toString());
        }

        private void otherInfos(Selectable node,Map<String,String> crawlerDataMap){
            List<String> targets = node.xpath(".//div[@class=\"purpose clearfix\"]/em/text()").all();
            crawlerDataMap.put("target", StringUtils.joinWith(",",targets.toArray()));

            List<String> images = node.xpath(".//ul[@class=\"photo_ul\"]//a/img/@src").all();
            crawlerDataMap.put("imageurl",StringUtils.joinWith("\\x01",images.toArray()));

            List<String> content = node.xpath(".//div[@class=\"review_post\"]/dl/*/text()").all();
            crawlerDataMap.put("content",StringUtils.joinWith("",content.toArray()));

            String like = node.xpath(".//a[@class=\"zan_ment\"]/text()").get();
            crawlerDataMap.put("like",like);

            String releaseTime = node.xpath(".//div[@class=\"publish\"]/text()").get();
            crawlerDataMap.put("releaseTime",releaseTime);

            List<String> spans = node.xpath(".//div[@class=\"publish\"]/span/@class").all();
            String isBest=null;
            for (String span : spans) {
                if (span.startsWith("ssen_")){
                    isBest = span;
                }
            }
            crawlerDataMap.put("isBest",isBest);

        }
    }

    static class TestForumRun implements Runnable{

        HttpClientDownloader downloader;
        HttpConfig httpConfig;
        
        Map<String,CompiledScript> compiledScriptMap;

        public TestForumRun(HttpClientDownloader downloader, HttpConfig httpConfig) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
            compiledScriptMap = new HashMap<>();
        }

        @Override
        public void run() {
//            String url = "https://www.xcar.com.cn/bbs/viewthread.php?tid=27030255&page=3470";
            try {
                initScript();
            } catch (ScriptException e) {
                e.printStackTrace();
            } catch (FileNotFoundException e) {
                e.printStackTrace();
            }

            try {
                String encodeStr = "2wLsthBdCigIBhIGeGhzLW1wGgUwLjAuMDACOgZtZXd0d29CBTEuMC4xWgQxMDA1EiMKCkFuZHJvaWQgNy4xLjISBE1JIDkaBlhpYW9taSjgAzDWBiJaChg2MDQwMzkwZTAwMDAwMDAwMDEwMDA4YmISIGQ3NDBmN2E3M2JkN2NkZmQ3YmNhNDY0NThmZTJiMmI4OhxvUFBVUzVRT25UMmxoZFlWYzZlaVQ5eXRNbU44KgIIATIcCFASGDYwNTcyMDJkMDAwMDAwMDAyMTAzNzc3YzoxCiQwNzYxOGRhYS1mZWEyLTQ3MjUtYjZiYi02NTg1NjE0NmI2MTkwElionb2n88LvAkpVGlNwYWdlcy9zZWNvbmRhcnkvY29tbWVudC9pbmRleD9mcm9tPWVuZ2FnZWJhciZpZD02MDU3MjAyZDAwMDAwMDAwMjEwMzc3N2Mmc2NlbmU9MTAwNQ==";
                compiledScriptMap.get("decode").eval();
                Invocable invocable = (Invocable) compiledScriptMap.get("decode").getEngine();
                String result = (String) invocable.invokeFunction("goog.crypt.base64.decodeString",encodeStr);
                System.out.println(result);
            } catch (ScriptException e) {
                e.printStackTrace();
            } catch (NoSuchMethodException e) {
                e.printStackTrace();
            }


//            String url = "https://newcar.xcar.com.cn/1/";
//            HttpRequest httpRequest = new HttpRequest();
////            String cookie = "nguv=c_160999737235228227110117218341611798; _Xdwuv=5ff69c3ce5e31; _PVXuv=5ff69c3df8614; __jsluid_s=49e976754d92ae5b3a1f9be9e2f407db; __jsluid_h=ff270f9a2198fb6530f6e9e06bae425a; __clickidc=161000895164367053; XCAR_SubCategory_Last=; XCAR_LastViewProducts=56a124a103a112a106a46a48a56a124a119a101a105a118a116a115a97a108a124a76a54a65a36842a22885a124a103a112a106a46a48a56a95a56; _ga=GA1.3.1167406058.1610953357; bbs_cookietime=31536000; _9755xjdesxxd_=32; YD00788855712789%3AWM_NI=F0m5YpD3rvoviVqwl4RBBN9CM9VtRUJQBcBVrt3ILPmsANjZB0rip4gNpCVtIU%2Fc4OM42whgiAa9YLwIqpOqzjHZRZMFdLoXu7MGO9TXEqRAdH4k%2FKuhRWqgOg%2FUVqFEbHA%3D; YD00788855712789%3AWM_NIKE=9ca17ae2e6ffcda170e2e6eeaaee3ca2b485b7ec79f8e78fb2c44f968b9abbf86ebcbefcdafc40ede9a097e42af0fea7c3b92a86f0a3d6dc7c88edbe97ee7eab8abf98ef40929e8eafbc498fe79a95d15f8a8f8e84c15eb2acfdaacf5cafad8fafe95282f0be8af340a592c0a4d5808b8be184b7458fecfe98eb6dab9fafdadc48b597bf91ce7bbab688bbd1528f9aacadee54b7e983a8ce3398888b92d14ba898fc8cae41b7b89eb8f46d8db9e1b6d57b869baea9c437e2a3; YD00788855712789%3AWM_TID=3CTDI2Lqq6lBRFUQVFMualftsz9%2BvOqb; gdxidpyhxdE=%2B3So6ZiIjuRn1BbHZA4ULVrPQe%5CoP1Xe7tph3IDugPZb7ddi4KX9vfZL6n7Hg7QOvNy3EYBdv099SgVS3kgQB7TZ%2F3LBvami8wrBN8xzloKq9deeTMhp%2FQDYbcyU1zzH%2F3bQj4l4l03aXcca5l%2Fd%5CY6JMYkZh64%2Fx7XoQD9AcyEu0TQR%3A1611139484030; ft_c_guide=1; _Xdwnewuv=1; bbs_abtest=b; bbs_abtest_yj2=b; bbs_abtest_yj3=b; fw_slc=1%3A1613988173%3B1%3A1613988700%3B1%3A1613988703%3B1%3A1613988718%3B1%3A1613988727; fw_clc=1%3A1613988143%3B1%3A1613988161%3B1%3A1613988702%3B1%3A1613988717%3B1%3A1613988906; fw_pvc=1%3A1613988138%3B1%3A1613988144%3B1%3A1613988703%3B1%3A1613988717%3B1%3A1613988911; fw_exc=1%3A1613988702%3B1%3A1613988703%3B1%3A1613988717%3B1%3A1613988919%3B1%3A1613988925; isRemoveCookie=1; Hm_lvt_53eb54d089f7b5dd4ae2927686b183e0=1611911542,1612492467,1613988139,1614043829; _locationInfo_=%7Burl%3A%22http%3A%2F%2Fsh.xcar.com.cn%2F%22%2Ccity_id%3A%22507%22%2Cprovince_id%3A%222%22%2C%20city_name%3A%22%25E4%25B8%258A%25E6%25B5%25B7%22%7D; _newLocationInfo=%7B%22url%22%3A%22http%3A%2F%2Fsh.xcar.com.cn%2F%22%2C%22city_id%22%3A%22507%22%2C%22province_id%22%3A%222%22%2C%20%22city_name%22%3A%22%25E4%25B8%258A%25E6%25B5%25B7%22%7D; uv_firstv_refers=https%3A//www.xcar.com.cn/bbs/viewthread.php%3Ftid%3D27030255; place_prid_lin=2; place_crid_lin=507; place_prname=%E4%B8%8A%E6%B5%B7%E5%B8%82; place_crname=%E4%B8%8A%E6%B5%B7%E8%BD%A6%E5%B8%82; __jsl_clearance_s=1614043832.928|0|1Qu7hLRMVnkbXI5mvA95VtM7w%2Bg%3D; iwt_uuid=55c322d6-dcca-4fb4-8174-19f8fadc38eb; Hm_lpvt_53eb54d089f7b5dd4ae2927686b183e0=1614043839; _Xdwstime=1614043836; zg_did=%7B%22did%22%3A%20%22176db52511822c-0749a9c4fbd349-c791039-100200-176db525119196%22%7D; zg_8f3d0255011c4bc5bae66beca6584825=%7B%22sid%22%3A%201614043829225%2C%22updated%22%3A%201614043839360%2C%22info%22%3A%201613988138906%2C%22superProperty%22%3A%20%22%7B%5C%22platform_type%5C%22%3A%20%5C%22PC%5C%22%2C%5C%22login_id%5C%22%3A%20null%2C%5C%22project_name%5C%22%3A%20%5C%22XCAR%5C%22%2C%5C%22login_status%5C%22%3A%200%7D%22%2C%22platform%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22%22%2C%22landHref%22%3A%20%22https%3A%2F%2Fnewcar.xcar.com.cn%2F%22%2C%22zs%22%3A%200%2C%22sc%22%3A%200%2C%22firstScreen%22%3A%201614043829225%2C%22cuid%22%3A%20%2218786832%22%7D";
////            Map<String, String> cookieMap = processCookie(cookie);
//            Map<String, String> cookieMap = new HashMap<>();
//            for(int i=0;i<100;i++){
//                List<Map<String,String>> crawlerDataList = new ArrayList<>();
//
//                httpRequest.setUrl(String.format(url,Math.random()));
//
//                httpRequest.addHeader("user-agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36");
////                httpRequest.addHeader("cookie",cookie);
//                HttpPage httpPage = downloader.download(httpRequest, httpConfig);
////                Html responseHtml = httpPage.getHtml();
//
//                String js = httpPage.getRawText().substring(httpPage.getRawText().indexOf("("), httpPage.getRawText().lastIndexOf(")")+1);
//                ScriptEngine engine = new ScriptEngineManager().getEngineByName("nashorn");
//                try {
//                    String eval = (String) engine.eval(js);
//                    eval =  eval.substring(eval.indexOf("=")+1, eval.indexOf(";"));
//                    System.out.println(eval);
//                    cookieMap.put("__jsl_clearance_s",eval);
//                    httpRequest.addHeader("cookie",processCookie(cookieMap));
//                    httpPage = downloader.download(httpRequest, httpConfig);
//
//
//                    Matcher mtAuthor = Pattern.compile("};go\\((.*?)\\)</script>").matcher(httpPage.getRawText());
//                    if (mtAuthor.find()){
//                        String filePathPrefix = "D:\\chance\\gitspace\\chance-crawler-development\\crawler-dev-bootstrap\\src\\main\\java\\com\\chance\\cc\\crawler\\development\\bootstrap\\xcar\\js\\";
//                        String cookieGen = mtAuthor.group(1);
//                        Json cookieGenJson = new Json(cookieGen);
//                        String ha = cookieGenJson.jsonPath("$.ha").get();
//
//                        compiledScriptMap.get(ha).eval();
//                        Invocable invocable = (Invocable) compiledScriptMap.get(ha).getEngine();
//                        String result = (String) invocable.invokeFunction("go",JSON.parseObject(cookieGen, Map.class));
//
//                        result =  result.substring(result.indexOf("=")+1, result.indexOf(";"));
//                        System.out.println(result);
//                        cookieMap.put("__jsl_clearance_s",result);
//                        httpRequest.addHeader("cookie",processCookie(cookieMap));
//                        httpPage = downloader.download(httpRequest, httpConfig);
//
//                        String dataInfo = articleDataInfo(httpPage);
//                        System.out.println("Result:"+dataInfo);
//
//
//                    }
//
//                    System.out.println(httpPage.getRawText());
//
//                } catch (ScriptException e) {
//                    e.printStackTrace();
//                } catch (NoSuchMethodException e) {
//                    e.printStackTrace();
//                }
//
//
//                System.out.println(httpPage.getStatusCode() + "-->" + httpPage.getRawText());
//                Json dataInfoJson = new Json(dataInfo);
//                String author_name = dataInfoJson.jsonPath($_type + ".author_name").get();
//                String author_id = dataInfoJson.jsonPath($_type + ".author_id").get();
//                String content_title = dataInfoJson.jsonPath($_type + ".content_title").get();
//                String content_id = dataInfoJson.jsonPath($_type + ".content_id").get();
//
//
//
//                List<Selectable> nodes = responseHtml.xpath("//div[@class=\"details\"]/div[@class=\"floor_div\"]").nodes();
//
//                for(Selectable node : nodes){
//
//                }
//                System.out.println(httpPage.getStatusCode() + "-->" + JSON.toJSONString(crawlerDataList));
//            }
        }

        private void initScript() throws ScriptException, FileNotFoundException {
            ScriptEngineManager sm = new ScriptEngineManager();
            NashornScriptEngineFactory factory = null;
            for (ScriptEngineFactory f : sm.getEngineFactories()) {
                if (f.getEngineName().equalsIgnoreCase("Oracle Nashorn")) {
                    factory = (NashornScriptEngineFactory)f;
                    break;
                }
            }
            String[] stringArray = new String[]{"-doe", "--global-per-engine"};
            ScriptEngine scriptEngine = factory.getScriptEngine(stringArray);
            List<String> ciphers = Arrays.asList("md5", "sha1", "sha256","decode");
            String filePathPrefix = "D:\\chance\\gitspace\\chance-crawler-development\\crawler-dev-bootstrap\\src\\main\\java\\com\\chance\\cc\\crawler\\development\\bootstrap\\xcar\\js\\";

            for (String cipher : ciphers) {
                String filePath = filePathPrefix+cipher+".js";
                final CompiledScript compiled = ((Compilable)scriptEngine).compile(new FileReader(filePath));
                compiledScriptMap.put(cipher,compiled);
            }
        }

        private String processCookie(Map<String,String> cookieMap){
            StringBuffer cookieSB = new StringBuffer();
            for (Map.Entry<String, String> entry : cookieMap.entrySet()) {
                cookieSB.append(entry.getKey())
                        .append("=")
                        .append(entry.getValue())
                        .append(";");
            }
            return cookieSB.substring(0,cookieSB.lastIndexOf(";"));
        }

        private Map<String,String> processCookie(String cookieStr){
            String[] cookieArrays = cookieStr.split(";");
            Map<String,String> cookieMap = new HashMap<>();
            for (String cookieArray : cookieArrays) {
                String[] cookieParts = cookieArray.split("=");
                if (cookieParts.length ==1 ){
                    cookieMap.put(cookieParts[0].trim(),"");
                } else if (cookieParts.length == 2){
                    cookieMap.put(cookieParts[0].trim(),cookieParts[1].trim());
                } else {
                    logger.error("unParse cookie {}",cookieArray);
                }
            }
            return cookieMap;
        }

        private String articleDataInfo(HttpPage page){
            String dataStr = "data: ";
            String dataInfo = null;
            Matcher mtAuthor = Pattern.compile(dataStr + "\\{([^}])*\\}").matcher(page.getRawText());
            if (mtAuthor.find()){
                dataInfo = mtAuthor.group();
                dataInfo = dataInfo.substring(dataInfo.indexOf(dataStr)+dataStr.length());

            }
            return dataInfo;
        }
    }

    static class TestGzipRun implements Runnable{

        HttpClientDownloader downloader;
        HttpConfig httpConfig;

        public TestGzipRun(HttpClientDownloader downloader, HttpConfig httpConfig) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
        }

        @Override
        public void run() {
            String url = "https://c.api.weibo.com/2/search/statuses/historical/download.json?source=1428199813&" +
                    "access_token=2.00o4_w1HrAaeYBedf38e38b8SnITmD&" +
                    "task_id=141902626&timestamp=1611726555&signature=9e379837e063c87cff1469819461ff21";
            HttpRequest httpRequest = new HttpRequest();

            for(int i=0;i<1;i++){
                List<Map<String,String>> crawlerDataList = new ArrayList<>();

                httpRequest.setUrl(url);
                httpConfig.setResponseTextGenerateHtml(false);
                HttpPage httpPage = downloader.download(httpRequest, httpConfig);
                try {
                    IOUtils.write(httpPage.getBytes(),new FileOutputStream("C:\\Users\\Administrator\\Desktop\\jar\\a.zip"));
                } catch (IOException e) {
                    e.printStackTrace();
                }

                System.out.println(httpPage.getStatusCode() + "-->" + JSON.toJSONString(crawlerDataList));
            }
        }

        private String articleDataInfo(HttpPage page){
            String dataStr = "data: ";
            String dataInfo = null;
            Matcher mtAuthor = Pattern.compile(dataStr + "\\{([^}])*\\}").matcher(page.getRawText());
            if (mtAuthor.find()){
                dataInfo = mtAuthor.group();
                dataInfo = dataInfo.substring(dataInfo.indexOf(dataStr)+dataStr.length());

            }
            return dataInfo;
        }
    }
}
